// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <xnnpack/assembly.h>

.syntax unified

// void xnn_f32_gemm_minmax_ukernel_4x4__aarch32_vfp_ld64(
//     size_t mr,                            r0
//     size_t nc,                            r1
//     size_t kc,                            r2 -> r5
//     const uint8_t*restrict a,             r3
//     size_t a_stride,          sp + 96  -> (r11)
//     const void*restrict w,    sp + 100 -> r9
//     uint8_t*restrict c,       sp + 104 -> r6
//     size_t cm_stride,         sp + 108 -> (r7)
//     size_t cn_stride,         sp + 112 -> r11
//     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])  sp + 116 -> (r11)


// inner loop registers

// A0   r3  s0-s1  d0
// A1  r12  s2-s3  d1
// A2  r10  s4-s5  d2
// A3   r0  s6-s7  d3

// B    r9  s12, s13, s14, s15 d6-d7
// B        s10, s11, s12, s13 d5-d6

// C0   r6 s16-s17  d8  s18-s19  d9
// C1   r4 s20-s21 d10  s22-s23 d11
// C2   r8 s24-s25 d12  s26-s27 d13
// C3   r7 s28-s29 d14  s30-s31 d15

// Clamp (r5) s8, s9 d4

BEGIN_FUNCTION xnn_f32_gemm_minmax_ukernel_4x4__aarch32_vfp_ld64
        .arm
#ifndef __APPLE__
        .arch armv6
        .fpu vfp
#endif
        // Push 96 bytes
        PUSH   {r4, r5, r6, r7, r8, r9, r10, r11}  // 32
        VPUSH  {d8-d15}                            // +64 = 96

        LDR     r11, [sp, 96]         // Load a_stride
        LDRD    r6, r7, [sp, 104]     // Load c and cm_stride
        LDR     r5,  [sp, 116]        // Load params

        // Clamp A and C pointers
        CMP    r0, 2                 // if mr >= 2
        ADD    r12, r3, r11          //   a1 = a0 + a_stride
        ADD    r4, r6, r7            //   c1 = c0 + cm_stride
        MOVLO  r12, r3               // a1
        MOVLO  r4, r6                // c1

        LDR     r9, [sp, 100]        // Load w

                                     // if mr > 2
        ADD    r10, r12, r11         //   a2 = a1 + a_stride
        ADD    r8, r4, r7            //   c2 = c1 + cm_stride
        MOVLS  r10, r12              // a2
        MOVLS  r8, r4                // c2

        VLDR   d4, [r5]              // Load min/max values

        CMP    r0, 4                 // if mr >=4
        ADD    r0, r10, r11          //   a3 = a2 + a_stride
        ADD    r7, r8, r7            //   c3 = c2 + cm_stride
        LDR    r11, [sp, 112]        // Load cn_stride
        MOVLO  r0, r10               // a3
        MOVLO  r7, r8                // c3


0:
        # Load initial bias from w into accumulators
        VLDM        r9!, {d8-d9}   // Bias
        SUBS        r5, r2, 8
        VMOV.F64    d10, d8
        VMOV.F64    d12, d8
        VMOV.F64    d14, d8
        VMOV.F64    d11, d9
        VMOV.F64    d13, d9
        VMOV.F64    d15, d9
        BLO         3f               // less than 2 channels?

        // Main loop - 2 floats of A (8 bytes)
1:
        VLDM        r3!, {d0}        // A0
        VLDM        r9!, {d6-d7}     // B0
        VLDM       r12!, {d1}        // A1
        VLDM       r10!, {d2}        // A2
        VLDM        r0!, {d3}        // A3

        VMLA.F32    s16, s12, s0
        VMLA.F32    s17, s13, s0
        VMLA.F32    s20, s12, s2
        VMLA.F32    s21, s13, s2
        VMLA.F32    s24, s12, s4
        VMLA.F32    s25, s13, s4
        VMLA.F32    s28, s12, s6
        VMLA.F32    s29, s13, s6

        VMLA.F32    s18, s14, s0
        VMLA.F32    s19, s15, s0
        VMLA.F32    s22, s14, s2
        VMLA.F32    s23, s15, s2
        VLDM        r9!, {d5-d6}     // B1
        VMLA.F32    s26, s14, s4
        VMLA.F32    s27, s15, s4
        VMLA.F32    s30, s14, s6
        VMLA.F32    s31, s15, s6

        VMLA.F32    s16, s10, s1
        VMLA.F32    s17, s11, s1
        VMLA.F32    s20, s10, s3
        VMLA.F32    s21, s11, s3
        VMLA.F32    s24, s10, s5
        VMLA.F32    s25, s11, s5
        VMLA.F32    s28, s10, s7
        VMLA.F32    s29, s11, s7

        SUBS        r5, r5, 8

        VMLA.F32    s18, s12, s1
        VMLA.F32    s19, s13, s1
        VMLA.F32    s22, s12, s3
        VMLA.F32    s23, s13, s3
        VMLA.F32    s26, s12, s5
        VMLA.F32    s27, s13, s5
        VMLA.F32    s30, s12, s7
        VMLA.F32    s31, s13, s7

        BHS         1b

        // Is there a remainder?- 1 floats of A (4 bytes)
        TST         r5, 4
        BNE         3f

2:
        // Clamp
        VCMPE.F32   s8, s16
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s17
        VMOVPL.F32  s16, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s18
        VMOVPL.F32  s17, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s19
        VMOVPL.F32  s18, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s20
        VMOVPL.F32  s19, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s21
        VMOVPL.F32  s20, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s22
        VMOVPL.F32  s21, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s23
        VMOVPL.F32  s22, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s24
        VMOVPL.F32  s23, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s25
        VMOVPL.F32  s24, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s26
        VMOVPL.F32  s25, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s27
        VMOVPL.F32  s26, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s28
        VMOVPL.F32  s27, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s29
        VMOVPL.F32  s28, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s30
        VMOVPL.F32  s29, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s8, s31
        VMOVPL.F32  s30, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s16
        VMOVPL.F32  s31, s8
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s17
        VMOVMI.F32  s16, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s18
        VMOVMI.F32  s17, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s19
        VMOVMI.F32  s18, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s20
        VMOVMI.F32  s19, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s21
        VMOVMI.F32  s20, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s22
        VMOVMI.F32  s21, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s23
        VMOVMI.F32  s22, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s24
        VMOVMI.F32  s23, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s25
        VMOVMI.F32  s24, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s26
        VMOVMI.F32  s25, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s27
        VMOVMI.F32  s26, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s28
        VMOVMI.F32  s27, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s29
        VMOVMI.F32  s28, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s30
        VMOVMI.F32  s29, s9
        VMRS APSR_nzcv, FPSCR
        VCMPE.F32   s9, s31
        VMOVMI.F32  s30, s9
        VMRS APSR_nzcv, FPSCR
        VMOVMI.F32  s31, s9

        SUBS        r1, r1, 4
        BLO         4f

        // Store full 4 x 4
        VSTM        r6, {d8-d9}
        SUB         r0, r0, r2
        ADD         r6, r11
        VSTM        r4, {d10-d11}
        SUB         r10, r10, r2
        ADD         r4, r11
        VSTM        r8, {d12-d13}
        SUB         r12, r12, r2
        ADD         r8, r11
        VSTM        r7, {d14-d15}
        SUB         r3, r3, r2
        ADD         r7, r11
        BHI         0b

        VPOP        {d8-d15}
        POP         {r4, r5, r6, r7, r8, r9, r10, r11}
        BX          lr

3:
        // Remainder- 1 floats of A (4 bytes)
        VLDM         r3!, {s0}       // A0
        VLDM         r9!, {d6-d7}    // B
        VLDM        r12!, {s1}       // A1
        VLDM        r10!, {s2}       // A2
        VLDM         r0!, {s3}       // A3

        VMLA.F32    s16, s12, s0
        VMLA.F32    s17, s13, s0
        VMLA.F32    s18, s14, s0
        VMLA.F32    s19, s15, s0

        VMLA.F32    s20, s12, s1
        VMLA.F32    s21, s13, s1
        VMLA.F32    s22, s14, s1
        VMLA.F32    s23, s15, s1

        VMLA.F32    s24, s12, s2
        VMLA.F32    s25, s13, s2
        VMLA.F32    s26, s14, s2
        VMLA.F32    s27, s15, s2

        VMLA.F32    s28, s12, s3
        VMLA.F32    s29, s13, s3
        VMLA.F32    s30, s14, s3
        VMLA.F32    s31, s15, s3

        B           2b

        // Store odd width
4:
        TST        r1, 2
        BEQ        5f
        VSTM       r6!, {d8}
        VMOV.F32   s16, s18
        VSTM       r4!, {d10}
        VMOV.F32   s20, s22
        VSTM       r8!, {d12}
        VMOV.F32   s24, s26
        VSTM       r7!, {d14}
        VMOV.F32   s28, s30

5:
        TST         r1, 1
        BEQ         6f
        VSTR        s16, [r6]
        VSTR        s20, [r4]
        VSTR        s24, [r8]
        VSTR        s28, [r7]

6:
        VPOP        {d8-d15}
        POP         {r4, r5, r6, r7, r8, r9, r10, r11}
        BX          lr

END_FUNCTION xnn_f32_gemm_minmax_ukernel_4x4__aarch32_vfp_ld64

#ifdef __ELF__
.section ".note.GNU-stack","",%progbits
#endif
